{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tempfile\n",
    "import tensorflow as tf\n",
    "\n",
    "from tf_rl.controller import DiscreteDeepQ, HumanController\n",
    "from tf_rl.simulation import KarpathyGame\n",
    "from tf_rl import simulate\n",
    "from tf_rl.models import MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/tmp3rSEdd\n"
     ]
    }
   ],
   "source": [
    "LOG_DIR = tempfile.mkdtemp()\n",
    "print(LOG_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from random import randint, gauss\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "class DiscreteHill(object):\n",
    "    \n",
    "    directions = [(0,1), (0,-1), (1,0), (-1,0)]\n",
    "    \n",
    "    def __init__(self, board=(10,10), variance=4.):\n",
    "        self.variance = variance\n",
    "        self.target = (0, 0)\n",
    "        while self.target == (0, 0):\n",
    "            self.target   = (randint(-board[0], board[0]), randint(-board[1], board[1]))\n",
    "        self.position = (0, 0)\n",
    "    \n",
    "    @staticmethod\n",
    "    def add(p, q):\n",
    "        return (p[0] + q[0], p[1] + q[1])\n",
    "        \n",
    "    @staticmethod\n",
    "    def distance(p, q):\n",
    "        return abs(p[0] - q[0]) + abs(p[1] - q[1])\n",
    "    \n",
    "    def estimate_distance(self, p):\n",
    "        distance = DiscreteHill.distance(self.target, p) - DiscreteHill.distance(self.target, self.position)\n",
    "        return distance + abs(gauss(0, self.variance))\n",
    "        \n",
    "    def observe(self):    \n",
    "        return np.array([self.estimate_distance(DiscreteHill.add(self.position, delta)) \n",
    "                         for delta in DiscreteHill.directions])\n",
    "    \n",
    "    def perform_action(self, action):\n",
    "        self.position = DiscreteHill.add(self.position, DiscreteHill.directions[action])\n",
    "        \n",
    "    def is_over(self):\n",
    "        return self.position == self.target\n",
    "    \n",
    "    def collect_reward(self, action):\n",
    "        return -DiscreteHill.distance(self.target, DiscreteHill.add(self.position, DiscreteHill.directions[action])) \\\n",
    "            + DiscreteHill.distance(self.target, self.position) - 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception AssertionError: AssertionError() in <bound method InteractiveSession.__del__ of <tensorflow.python.client.session.InteractiveSession object at 0x7fa0300f42d0>> ignored\n"
     ]
    }
   ],
   "source": [
    "n_prev_frames = 3\n",
    "\n",
    "# Tensorflow business - it is always good to reset a graph before creating a new controller.\n",
    "tf.ops.reset_default_graph()\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "# This little guy will let us run tensorboard\n",
    "#      tensorboard --logdir [LOG_DIR]\n",
    "journalist = tf.train.SummaryWriter(LOG_DIR)\n",
    "\n",
    "# Brain maps from observation to Q values for different actions.\n",
    "# Here it is a done using a multi layer perceptron with 2 hidden\n",
    "# layers\n",
    "brain = MLP([n_prev_frames * 4 + n_prev_frames - 1,], [4], \n",
    "            [tf.identity])\n",
    "\n",
    "# The optimizer to use. Here we use RMSProp as recommended\n",
    "# by the publication\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate= 0.001, decay=0.9)\n",
    "\n",
    "# DiscreteDeepQ object\n",
    "current_controller = DiscreteDeepQ(n_prev_frames * 4 + n_prev_frames - 1, 4, brain, optimizer, session,\n",
    "                                   discount_rate=0.9, exploration_period=100, max_experience=10000, \n",
    "                                   store_every_nth=1, train_every_nth=4, target_network_update_rate=0.1,\n",
    "                                   summary_writer=journalist)\n",
    "\n",
    "session.run(tf.initialize_all_variables())\n",
    "session.run(current_controller.target_network_update)\n",
    "# graph was not available when journalist was created  \n",
    "journalist.add_graph(session.graph_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Game 9900: iterations before success 16. Pos: (6, 6), Target: (6, 6)\n"
     ]
    }
   ],
   "source": [
    "performances = []\n",
    "\n",
    "try:\n",
    "    for game_idx in range(10000):\n",
    "        game = DiscreteHill()\n",
    "        game_iterations = 0\n",
    "\n",
    "        observation = game.observe()\n",
    "        \n",
    "        prev_frames = [(observation, -1)] * (n_prev_frames - 1)\n",
    "        memory = np.concatenate([np.concatenate([observation, np.array([-1])])] * (n_prev_frames - 1) + [observation])\n",
    "        \n",
    "        while game_iterations < 50 and not game.is_over():\n",
    "            action = current_controller.action(memory)\n",
    "            if n_prev_frames > 1:\n",
    "                prev_frames = prev_frames[1:] + [(observation, action)]\n",
    "            reward = game.collect_reward(action)\n",
    "            game.perform_action(action)\n",
    "            observation = game.observe()\n",
    "            new_memory = np.concatenate([np.concatenate([a, np.array([b])]) for (a, b) in prev_frames] + [observation])\n",
    "            current_controller.store(memory, action, reward, new_memory)\n",
    "            current_controller.training_step()\n",
    "            memory = new_memory\n",
    "            game_iterations += 1\n",
    "            cost = abs(game.target[0]) + abs(game.target[1])\n",
    "        performances.append((game_iterations - cost) / float(cost))\n",
    "        if game_idx % 100 == 0:\n",
    "            print \"\\rGame %d: iterations before success %d.\" % (game_idx, game_iterations),\n",
    "            print \"Pos: %s, Target: %s\" % (game.position, game.target),\n",
    "except KeyboardInterrupt:\n",
    "    print \"Interrupted\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa074c318d0>]"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEACAYAAABMEua6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGzdJREFUeJzt3XuUFPWd9/H3F2aG4X4xgiA35RijBCFuRPGymcQbxLia\nk+yqJ0qSx71kicTk8WxQkydgTtYVJXHDusnj8bau8QmwrCis4CWRUdYrLiLITVQMcr8Ml5GLzDDf\n549ftz0DM9M9M91d3dWf1zl9qrqquupbv+n59q9+9asqc3dERCQeOkUdgIiIZI+SuohIjCipi4jE\niJK6iEiMKKmLiMSIkrqISIykTepm1sfM5prZGjNbbWbn5SMwERFpu7IMlvk1sNDdv2lmZUD3HMck\nIiLtZK1dfGRmvYG33P3U/IUkIiLtla755RRgp5k9YmbLzOwBM+uWj8BERKTt0iX1MuBs4DfufjZw\nALg151GJiEi7pGtT3wRscvelifdzOSapm5luHiMi0g7ubtleZ6s1dXffBnxkZp9NTLoEWNXMcnq5\nM3Xq1MhjKJSXykJlobJo/ZUrmfR+mQw8bmYVwPvAd3MWjYiIdEjapO7ubwPn5CEWERHpIF1RmkVV\nVVVRh1AwVBYpKosUlUXutdpPPaMVmHku24dEROLIzPB8nygVEZHikvWk/tprsHBhttcqIiKZyHrz\niyUOJtQiIyLSMjW/iIhIWllN6ps3Z3NtIiLSVplcfJSRK66AbdtS72tqoF+/bK1dREQykZWa+qFD\n4eTosmWpaQsWZGPNIiLSFllJ6vPmNX0/ZoyaYkREopCVpF5fDxUVYXzBArj4Yigvz8aaRUSkLbKS\n1L/9bThyJIx37gzdu8Ojj8LOndlYu4iIZCrrXRo//hhWrIBVq+C667K9dhERaU1WLj4Cp7ISZs+G\nr34VeveGgwdh+HDYsCE7gYqIxEmuLj7KWlL/8EMYNixM27EjnCzdulVXloqINKfgryhNJnSA/v1h\nzpww/uGH2dqCiIikk5WkvmPH8dOGDg3DNWuysQUREclEVpL6iScePy2Z1B99NBtbEBGRTOT0IRnJ\nOzbu2we9enVoMyIisVLwberNmTEjDH/3u1xuRUREkrJ2Q6/m3HILPP88NDTkcisiIpKU82eU6qEZ\nIiLHK8rml8aefTZfWxIRKV05T+p33BGG48eH55eKiEju5Dyp/+xnoW0d4PBh2LVLTTEiIrmSl+aX\ne+4J94Gprw992nv0yMdWRURKT16SuhmMHAmXXhreHzyYj62KiJSevJ0oPbZb47p1+dqyiEjpyKif\nupl9COwHjgJ17j62rRtatCgM3UPNXbV1EZHsy/TiIweq3L0mGxs96yzo1OgYYdOmcCuBkSOzsXYR\nkdLVluaXDnWSf+EFePXVML5iBXz/+/DRR+H9NdfA5z/fkbWLiAhkeEWpmX0A7CM0v9zv7g80mtfq\nFaXNry81fvRoeK4pqKujiJSOqK8ovcDdvwBMAL5vZhd1ZKPz5qXGDx3qyJpERKSxjNrU3X1rYrjT\nzOYBY4ElyfnTpk37dNmqqiqqqqpaXd/VV8OECfDyy6mHaIwb17bARUSKSXV1NdXV1TnfTtrmFzPr\nBnR291oz6w48B9zh7s8l5re5+QWgrg4qKmD0aHj77TDtmWfg8svbvCoRkaITZfPLAGCJmS0HXgf+\nK5nQO6K8PAzffhsmTYKuXdXNUUSko9I2v7j7BmBMLoPo3Ts0x+i+6yIiHZO3K0qbc999YThiROi3\nPn16at7994eXiIhkLqdPPkqnoiIMKyth7twwbhZq7N/7Xnj/d38XTWwiIsUo0pp6bW0Ydu0Ka9em\npr/ySjTxiIgUu0iTerI747vvwumnQ03iJgQXXgg9e4bx996LJjYRkWKU82eUtmbLFrjoonDbgO7d\nk+truswrr6gPu4jET666NEaa1JtfX2q8V6/wGLzZs7O2ehGRghD1bQLy5q/+KjX+uc/BnDnRxSIi\nUmwKrqYO8PDDcOONsHcv9OkT7g9TWZnVTYiIRKpkml+OX38YNjQc394uIlKsSqb55Vh9+4bhm29G\nG4eISDEo+KR+zz1huHp1tHGIiBSDgm9+AZg8OdxSYOdO+MxncropEZG8KNnmF4Dt28Pw/fejjUNE\npNAVRVKfNQsuuQQ2bIg6EhGRwlYUSb1TJ+jfH371q6gjEREpbEWR1AFuuAGWLo06ChGRwlY0Sf3c\nc8PDNEREpGVFk9T79AF32L076khERApX0SR1Mxg1CpYtizoSEZHCVRT91FPbCsM8bU5EJGdKup96\n0s9+Fu6/LiIizSuqpD5mTOpeMCIicryiSuqdOsH8+VBfH3UkIiKFqaiS+vjxYZh8tqmIiDRVVEm9\nSxcYOhQeeCDqSEREClNRJXWA889XTV1EpCVFl9S/8Q3o2TPqKEREClPRJfVhw2DjxqijEBEpTEV1\n8RGEe6ufdJKeWSoixS3Si4/MrLOZvWVmC7IdQFsNGAAVFfCnP0UdiYhI4cm0+eVmYDVQEBfon302\nLFoUdRQiIoUnbVI3s8HAV4EHgYJo8Ni8GSZNijoKEZHCk0lN/V7gH4CGHMeSsd/9Ds45J+ooREQK\nT1lrM83sa8AOd3/LzKpaWm7atGmfjldVVVFV1eKiWXHmmbByZU43ISKSVdXV1VRXV+d8O632fjGz\nO4EbgHqgEugF/Ke7T2y0TF57v0C49W6nTqEZZtCgvG5aRCQrIun94u63u/sQdz8FuBZ4oXFCj4oZ\nDB8O77wTdSQiIoWlrRcfFUTvF4CdO2HWrKijEBEpLBkndXd/0d3/IpfBtMXDD0NNTdRRiIgUlqK7\nTUDSF7+o55WKiByr6G4TkFRXB927w4EDUF6e982LiHSInlF6jPLykNiXL486EhGRwlG0ST1p8uSo\nIxARKRxFndQnToTXX486ChGRwlG0beoAe/dC377hQdSdO0cSgohIu6hNvRl9+oRb8W7bFnUkIiKF\noaiTOsC+fVBbG3UUIiKFoeiTerdusH591FGIiBSGok/ql10G+/dHHYWISGEo+qTevz/s2BF1FCIi\nhaHok/rJJ4db8IqIiJK6iEisFH1SHzwYNm2KOgoRkcJQ9EldNXURkZSivqIU4NChcBHSoUPhEXci\nIsVAV5S2oGtXqKjQBUgiIhCDpA7QpQt88knUUYiIRE9JXUQkRmKR1CsrldRFRCBGSf3QoaijEBGJ\nXiyS+sknq6+6iAjEJKn37RtuwSsiUupikdR79dKdGkVEIEZJfe/eqKMQEYleLJL65z8Py5dHHYWI\nSPRikdT791ebuogIxCSpq0ujiEiQNqmbWaWZvW5my81stZn9Uz4Ca4vKSjh8OOooRESiV5ZuAXc/\nbGZfdveDZlYG/LeZXeju/52H+DLSrRscPBh1FCIi0cuo+cXdkymzAugM1OQsonbo3x+2b486ChGR\n6GWU1M2sk5ktB7YDi919dW7DapsBA2D3bqirizoSEZFopW1+AXD3BmCMmfUGnjWzKnevTs6fNm3a\np8tWVVVRVVWV3SjTKCsLtfWtW2Ho0LxuWkQkI9XV1VRXV+d8O21+8pGZ/R/gkLvPSLyP9MlHSePG\nwYwZcMEFUUciIpJeZE8+MrPPmFmfxHhX4FLgrWwH0lF6ALWISGbNLwOBR82sE+FH4DF3/2Nuw2q7\nYcNgw4aooxARiVYmXRpXAmfnIZYOOessWLQo6ihERKIViytKAcaMgbcKrlFIRCS/2nyi9LgVFMiJ\n0ro66N0bduyAHj2ijkZEpHWRnSgtFuXlcPrpsHZt1JGIiEQnNkkd4LTTYP36qKMQEYlOrJL66afD\nunVRRyEiEp1YJfXdu+GOO6KOQkQkOrFK6mPHRh2BiEi0YtP7BWDPHujXD2pqoG/fqKMREWmZer9k\nIJnI778/2jhERKISq6QOcNttcOBA1FGIiEQjdkl93Dh49dWooxARiUbskvro0boASURKV6xOlEJ4\nAHXXrrB/P/TsGXU0IiLN04nSDFVWhqFuwysipSh2SR3g0kthy5aooxARyb9YJvXu3WHJkqijEBHJ\nv1gm9TPOgJ07o45CRCT/YpnUhw+HAjp3KyKSN7FM6pWVoReMiEipiWVS79Il3AdGRKTUxDKpNzTA\n00/rmaUiUnpid/ERhPb0Tp3g2mvh97+POhoRkePp4qM2MIM774Q+faKOREQkv2KZ1AGGDYO9e6OO\nQkQkv2Kb1Lt1gzfeiDoKEZH8im1SHzwYPvgg6ihERPIrlidKIfRT7907DC3rpyJERDpGJ0rbqLIy\nJHNdhCQipSRtUjezIWa22MxWmdk7ZvaDfASWDb16hfuqi4iUikxq6nXAj9x9JHAe8H0zOyO3YWVH\nz55QWxt1FCIi+ZM2qbv7Nndfnhj/GFgDDMp1YNlQUaFujSJSWtrUpm5mw4EvAK/nIphsq6+HFSui\njkJEJH8y7v1iZj2AauAX7v5ko+k+derUT5erqqqiqqoqu1G2U+fO4T4wBdg5R0RKTHV1NdXV1Z++\nv+OOO3LS+yWjpG5m5cB/AYvc/Z+PmVeQXRoB7r8fvvc9ePddOO20qKMREUmJrEujmRnwELD62IRe\n6P76r8PwpZeijUNEJF/S1tTN7ELgJWAFkFz4Nnd/JjG/YGvqAP36hXurF3CIIlKCclVTj+0VpUn7\n94crS5cvh9Gjo45GRCTQFaXt1KsXfP3roV1dRCTuYp/UIdzca8uWqKMQEcm9kkjqAwfC5s1RRyEi\nknslkdQ/9zn1gBGR0hD7E6UQ7qs+YgRs3w79+0cdjYiITpR2yKmnwoQJ8MADUUciIpJbJZHUAa6+\nWj1gRCT+Siapn3mmkrqIxF9JtKkD7NgBAwbArl1wwglRRyMipU5t6h3Uvz/07Qvz5kUdiYhI7pRM\nUodw18Y5c6KOQkQkd0qm+QXgwAHo0QOWLYPPfha6d486IhEpVbqhV5ZYoyIsorBFJGbUpp4lf/mX\nqfE//CG6OEREcqHkauoQHkb9ta/B2rWhV0ynkvtpE5Goqfkly5YuhbFjw3hDQ9NmGRGRXFPzS5ad\ncw68+WYYf+21kNSTr6VLo41NRKS9SramntSzJ3z88fHT6+qgrCz/8YhIaVBNPUeWLAnDe++F2lpY\nvbrpdBGRYlLyNfXm3HcfTJ4Mt9wCM2ZEHY2IxJFOlOZRfT2MHBluABazXRORAqHmlzwqK4OVK8P4\nyy9HG4uISFsoqbegogJuvBEuvTTU3EVEioGaX1qxbx/06RPuF1NbG3U0IhInan6JQO/ecOutocvj\nc8+FaUruIlLIVFNP4+jR5vurb9wIQ4bkPx4RiQfV1CPSuXO4jcDJJzedfu+90cQjItIaJfUMmMGm\nTaF7o3u4u+O994bpV14ZdXQiIilpm1/M7GHgCmCHu49qZn6sm19a0twNwHRjMBHJVJTNL48A47O9\n4WKXrLUvWpSa1qkTXHUVvPNOdHFJ9tXV5fYitK1bYf/+MF5fDy+8ECoIIu2RNqm7+xJgTx5iKUrj\nx4d/+HHjwvv582HUqFBjX7gQnn8e7rwzPEpPcsc93Er5O98JTWUnnBD+BsmLyFrS0ABHjsD//E9Y\nfvp0OPHEMP7b38LBg+GahU6dYMUKuPnmsO7rrgv35W/rNQy1tamE/cYbcNddMGhQ6Gn1538O5eVw\n8cXhXM6kSXDPPTBlCtx+ezhpL5KWu6d9AcOBlS3Mc3Gvq3PfsycMn3zSvWfPZF2+6etv/iYsE6WG\nBvf33w9xbtjQ8nI1Ne6XXea+fn1u4zl61P3RR90nTgxl9MknqXnf+laYVlvrPmGCe9eu7nv3pubP\nn+8+b577r37VfHmD+3vvhWU//ND9wIGw/y++6P7gg80vX17e8rqaez3xhPuYMe5TpqTf10zW9+tf\ntzyvcdlIcUvkzoxycFteSuo59NFH7vv2ua9d2/w/6B//GBLamjXuf/pTx7d33XXuZWXuu3alpu3c\nmdreTTe5z5rVfCwff5z6TEND88usWRPmv/CCe3V1x+N1dz98uPltXXRRiKktyXXgwLDvTzwR9mHX\nrtS87t1b/tyYMe5nnOHeo0d4n9y3F190nzQpxOju/tJL4YewocF99Wr3QYOaT/DN2b8/tcy554bh\n5Mnuhw6llqmpCT88yXJZvjzM37XL/ec/T31+5MjslH22LFuWWUWlutp99+7wPaqpyX1cha6gk/rU\nqVM/fS1evDinBVHsFi8OpX7mmccnhLvuanstvqEh1N5efbXpuv7xH1tPgA8+6L5li/uzz6amLVoU\n1nn99eF9nz6hRv+jHzW/jl/+8vhYDh92v/nmMP/ll8P0l192X7Uq/ICdf37z65o71/3gwfBDeNNN\nTedt2eJ+yinhx+TgQfdTT00lx3/5F/d169yHDGm+prx7d2o9v/iF+0MPuT/+ePgRfegh94ULmy6/\ncqV7fX3m5V9b6z56tPu//7t7r17+aU37yBH3GTNCmQ4dmorhxz/OfN3Huuee1HoKxSWXpGJ68cWm\n8xoa3KdNa/k72KOH+9atHY9h507373zH/cor3f/+71Pr//rX3Z96qumRXZQWL17cJFcWdFKX9kl+\n+WbOdD/ppNT7zp3dN20KyzQ0hET/29+GeU8+GZLcsmXuTz/d9J/km98MCenYf553302ta+fO4+PY\ns6fp9iE0aRzrscfCvFtucZ86NYx/61uhRplJTXrs2OanJ2vCjU2f7n7BBU1rsoXu6NHW9z8bR2NH\nj7pXVobvSLbW2R6bN6f265ZbUuMTJ4bvSfL7Cu5/+7fuZuGoZ+ZM94svTs077bTwA9ge69eHWn8m\n3z0IR7L/9m/hyOzxx7NbHu0RWVIHfg9sAT4BPgK+e8z8nO98XDU0pL7Q9fWhptP4SzhiROZf2K98\nJfzDu4cmn2Stuy2uuCKsa8KEzJafPv34OKZMcb/qqjD/yivDtFGjUvNXr257XMUmua//8R+hLHfv\nzu76Z85sWuannx6O1JJHGPlod//GN8K2k812+/Yd/12YOLFpU+CxfvCD1LLXXOM+e3aIvaEhNHXN\nnu2+YEGovKxbF5b74IOwzEsvNd3W3LnHr7+hIfz4jBqVOoo69vX++2G5KERaU291BUrqWVVfHw7p\nk1+6QYNC+/KhQ+HLt2FDaDa44gr3f/3XqKMNRxTJtvbWHD3qvm1b7uMpBAsXtr/22RaHD7vfcEPz\nyerqq8OwS5fQzj9njvvGjaFtv7kk9tRToSad/PxPfxqOBhtLJvBk+35ztd2jR8PnHnkks2R5+HDr\n5ztae/XsGWremVq1yv3558N446PcQYNCs16+5Sqp694vIkWupgZ++lN48MHQpz6dgQNh2bLQdbNz\n5/D5E05oftkvfxl++MNwY7s1a1LTH3sMrr8+O/FDiHvJEnj6aRg+HM46K4wPGBC6k3btGm6FXVEB\nffvCnj1QXQ1f+lL7tzlzZuiimrRsGcyaBXffnZr2k5/Az38etveTn4RrCH7zm1AWN94I3bq1f/t6\n8pGIZMQ9dWVzbS188gk8/DD07x+upxg7NnWx0333wU03wamnhttfDB8ePl9fD0uXwoUXptY7eHC4\nBuDEE2HHjrzvVs5s3AjDhrX9c/X14UexvZTURSQrDh+GZ5+Fq69OTduyJdTgj3XkCOzaFS6QirO6\nunAl+LBh0K9fanp9PVx+eThauPvucAQxfTqMGAHXXNOxbSqpi0jWvfIKnHtux2qc0j5K6iIiMaL7\nqYuISFpK6iIiMaKkLiISI0rqIiIxoqQuIhIjSuoiIjGipC4iEiNK6iIiMaKkLiISI0rqIiIxoqQu\nIhIjSuoiIjGipC4iEiNK6iIiMaKkLiISI0rqIiIxoqQuIhIjSuoiIjGipC4iEiNK6iIiMaKkLiIS\nI0rqIiIxoqQuIhIjaZO6mY03s7Vmtt7MpuQjKBERaZ9Wk7qZdQbuA8YDZwLXmdkZ+QisGFVXV0cd\nQsFQWaSoLFJUFrmXrqY+FnjP3T909zpgFnBV7sMqTvrCpqgsUlQWKSqL3EuX1E8GPmr0flNimoiI\nFKB0Sd3zEoWIiGSFubect83sPGCau49PvL8NaHD36Y2WUeIXEWkHd7dsrzNdUi8D1gEXA1uAN4Dr\n3H1NtgMREZGOK2ttprvXm9lNwLNAZ+AhJXQRkcLVak1dRESKS4euKI37hUlmNsTMFpvZKjN7x8x+\nkJjez8yeN7N3zew5M+vT6DO3JcpjrZld1mj6n5nZysS8X0exP9lgZp3N7C0zW5B4X5JlYWZ9zGyu\nma0xs9Vmdm4Jl8Vtif+RlWb2/8ysS6mUhZk9bGbbzWxlo2lZ2/dEWc5OTH/NzIalDcrd2/UiNMe8\nBwwHyoHlwBntXV8hvoCTgDGJ8R6E8wtnAHcDP05MnwLclRg/M1EO5YlyeY/U0dAbwNjE+EJgfNT7\n184y+d/A48D8xPuSLAvgUeB/JcbLgN6lWBaJ/fkA6JJ4Pxv4dqmUBXAR8AVgZaNpWdt3YBLwm8T4\nNcCstDF1YGfGAc80en8rcGvUhZzjP+CTwCXAWmBAYtpJwNrE+G3AlEbLPwOcBwwE1jSafi3wf6Pe\nn3bs/2DgD8CXgQWJaSVXFokE/kEz00uxLPoRKjt9CT9uC4BLS6ksEgm6cVLP2r4nljk3MV4G7EwX\nT0eaX0rqwiQzG074RX6d8Afbnpi1HRiQGB9EKIekZJkcO30zxVlW9wL/ADQ0mlaKZXEKsNPMHjGz\nZWb2gJl1pwTLwt1rgF8CGwk95Pa6+/OUYFk0ks19/zTPuns9sM/M+rW28Y4k9ZI5w2pmPYD/BG52\n99rG8zz8hMa+LMzsa8AOd38LaLZvbamUBaHGdDbhsPhs4ADhSPVTpVIWZjYC+CGhtjoI6GFm1zde\nplTKojlR7HtHkvpmYEij90No+msTC2ZWTkjoj7n7k4nJ283spMT8gcCOxPRjy2QwoUw2J8YbT9+c\ny7hz4HzgL8xsA/B74Ctm9hilWRabgE3uvjTxfi4hyW8rwbL4IvCKu+9O1CSfIDTNlmJZJGXjf2JT\no88MTayrDOidODpqUUeS+pvAaWY23MwqCI348zuwvoJjZgY8BKx2939uNGs+4WQQieGTjaZfa2YV\nZnYKcBrwhrtvA/YnekgYcEOjzxQFd7/d3Ye4+ymENr8X3P0GSrMstgEfmdlnE5MuAVYR2pNLqiwI\n7cfnmVnXxD5cAqymNMsiKRv/E081s65vAn9Mu/UOniCYQDhJ8h5wW9QnLHJwAuRCQvvxcuCtxGs8\n4eTQH4B3geeAPo0+c3uiPNYClzea/mfAysS8mVHvWwfL5Uuker+UZFkAo4GlwNuE2mnvEi6LHxN+\n1FYSegWVl0pZEI5atwBHCG3f383mvgNdgDnAeuA1YHi6mHTxkYhIjOhxdiIiMaKkLiISI0rqIiIx\noqQuIhIjSuoiIjGipC4iEiNK6iIiMaKkLiISI/8fTk8JmejJIq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa028361e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "N = 500\n",
    "smooth_performances = [float(sum(performances[i:i+N])) / N for i in range(0, len(performances) - N)]\n",
    "\n",
    "plt.plot(range(len(smooth_performances)), smooth_performances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.colorbar.Colorbar instance at 0x7fa050305dd0>"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAJBCAYAAABRfpDFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGgJJREFUeJzt3X+wZGV95/HPhzsgDojEUAGBSU3WRYNGUYpfhUm8GLAm\nxJ8pE0MkGkxZVrJjTKqSqCQxQ2o3WZO4IVk2LiKiBCOpoBgMjmRivCpRR0aZAWeGBTRUZoZlQFkw\ngETuzHf/6AYu95zb/fTTp/ucnuf9quqy+/bpc773yvnO5zzn6XMcEQKAYQ5quwAAs4FmASAJzQJA\nEpoFgCQ0CwBJaBYAktAsgBlk+1Dbm21vtb3D9h/XLPNG29ts32L7X2y/aJxtrhrnwwDaERGP2j4r\nIh6xvUrSjbZ/PCJuXLLYtyT9ZEQ8aHudpA9IOiN3mzQLYEZFxCP9p4dImpN0/7L3v7zk5WZJx4+z\nPQ5DgBll+yDbWyXtlfS5iNgxYPFfkfTpcbZHswBmVETsj4gXq5cYftL2fN1yts+S9BZJ7xxnexyG\nABlsT/1LVRHhFX7+oO3rJZ0iaWHpe/1BzcskrYuI/zfO9mkWQKYNLW7L9lGSFiPiAdtPl3SOpIuW\nLfPDkj4h6fyIuHPcGmgWwGx6tqSP2D5IveGEv46Iz9p+myRFxKWS3iPpByS937YkPRYRp+Vu0HxF\nHRid7fivU9ze72nlw5BpYYATQBIOQ4BMB7ddwJSRLAAkIVkAmUrbeUgWAJKU1hyBxjBmAQA1SBZA\nptJ2HpIFgCSlNUegMYxZAEANmgWAJByGAJlK23lIFgCSlNYcgcYwwAkANUgWQKbSdh6SBYAkpTVH\noDGMWQBADZIFkIlkAQA1SBZAptJ2HpIFgCQ0CwBJSktSQGMY4ASAGiQLIFNpOw/JAkCS0poj0BjG\nLACgBskCyFTazkOyAJCktOYINIYxCwCoQbMAkITDECBTaTsPyQJAkk42C9vrbN9m+w7b72y7nmFs\nf8j2Xtu3tl1LKttrbH/O9nbb37D9623XNIjtQ21vtr3V9g7bf9x2TQdP8dEFnWsWtuckXSJpnaTn\nSzrP9ontVjXUFerVO0sek/SbEfECSWdI+i9d/jtHxKOSzoqIF0t6kaSzbP94y2UVpYuHXadJujMi\n7pIk21dLeo2knW0WNUhEfNH22rbrGEVE3CPpnv7zh2zvlHSsuv13fqT/9BBJc5Lub7GcTu48k9S5\nZCHpOEm7lrze3f8ZJqTf6F4iaXO7lQxm+yDbWyXtlfS5iNjRdk0l6WJzjLYLKIntwyVdI+kdEfFQ\n2/UMEhH7Jb3Y9jMl3WB7PiIW2qqnK2MJ09LFZLFH0polr9eoly7QMNsHS/q4pKsi4pNt15MqIh6U\ndL2kU9qupSRdTBZbJJ3Qj8Z3S3qDpPPaLOhAZNuSLpe0IyIubrueYWwfJWkxIh6w/XRJ50i6qM2a\nSBYti4hFSesl3SBph6S/jYjODrpJku2PSfqSpOfa3mX7grZrSvBSSeerd1bh5v6jy2d0ni3pn/tj\nFpslfSoiPttyTUVxBEMEwKhsxz1T3N4xkiLCU9xkReeSBYBuolkASNLFAU5gJhw8zb1ncYrbWgHJ\nAkASkgWQaVVhyWLsX9c2p1NwwGj7jEOXNdQb/6CZ1VQsSJqfwHrXTmCdkvRJSa+d0LpPntB63y/p\nVye07r0TWu+Vkt40gfW+YqSlD56bQAkdxpgFgCSMWQCZpjpm0QEdTxZr2y5gRD/adgEZZvG7WCe1\nXUCROt4b17ZdwIhmsVmc2nYBGbrRLKY6z6IDOp4sAHQFzQJAksKCFNAgTp0CQBXJAshV2N5DsgCQ\npLDeCDSosL2HZAHMoFFuP2n7VNuLtn92nG0W1huBBrW79zx++8mt/Xu/fM32puUXt+7fDvS9kj4j\naaxv1JIsgBkUEfdExNb+84fUu+3ksTWLvl29m0jdN+42SRZAro7Ms1jp9pO2j1PvPsEvV29e/1jX\nnhnaLPr3krhYvT/NByPiveNsEMBwC49IC98bvtyQ209eLOldERH9m0qNdRgy8L4h/eOd/yPpbPVu\nK3iTpPOWHhf1rpQ1qYvfTMratgvIMKmL30zSpC5+MymvSL5Slu2IEyddz5Lt7axexat/+8l/kLSx\n7q5ytr+lJxvEUZIekfTWiLgup4ZhyeI0SXdGxF39jV+tXqzp9B3CgANdyu0nI+I/LVn+CvXu4pbV\nKKThzeI4SbuWvN4t6fTcjQFozOO3n7zF9s39n10o6YclKSIubXqDw5oFF+MFVtLi6YGIuFEjnM2M\niLHvvzvs190jac2S12vUSxfLLCx5vlazOSaA8mzrP5BiWLPYIumE/qmZuyW9QdJ51cXmm60KmIqT\n9NSrbl012sc7cup0WgY2i4hYtL1e0g3q/WkuXz5DDEAZhh51RcRGSRunUAswWwqb0sh0bwBJCuuN\nQIMK23tIFgCSFNYbgQYVdjaEZAEgCc0CQBIOQ4Bche09JAsASQrrjUCDCtt7SBYAkhTWG4EGFbb3\nkCwAJCmsNwINYlIWAFSRLIBche09JAsASQrrjUCDCtt7SBYAkjTSG4+PNzaxmqnZ/boT2i5hZD99\n7SfaLmFkP6jvtF3CSK4a6+Z+B77CghTQIE6dAkAVyQLIVdjeQ7IAkKSw3gg0qLC9h2QBIElhvRFo\nEGdDAKCKZAHkKmzvIVkASFJYbwQaVNjeQ7IAkIRmASBJYUEKaFBhew/JAkCSwnoj0CAmZQFAFckC\nyFXY3kOyAJCksN4INKiwvYdkASBJYb0RaBBnQwCgimYBIAmHIUCuwvYekgWAJIX1RqBBhe09JAsA\nSQrrjUCDCtt7SBYAkhTWG4EGMSkLAKpIFkCuwvYekgWAJIX1RqBBhe09JAsASWgWAJIUFqSABnHq\nFACqSBZArsL2HpIFgCQ0CyDXqik+lrH9Idt7bd+6Unm2523fbPsbthfG/XVpFsBsukLSupXetH2k\npP8l6VUR8WOSXj/uBhs56nqP/rCJ1UzN3LX72i5hZG954cfaLmFkD2+ZrX+Lrhr1Ay2eDYmIL9pe\nO2CRX5T08YjY3V/+2+Nuc7b+3wSQ6gRJz7L9OdtbbP/SuCssbDwXaFC3956DJZ0s6ackrZb0Zdtf\niYg7clfY7V8XKNTCzt5jDLskfTsivifpe7a/IOkkSTQL4EAyf2Lv8biLrh15FX8v6RLbc5KeJul0\nSf9jnJpoFkCuFvce2x+T9DJJR9neJekP1Dv0UERcGhG32f6MpFsk7Zd0WUTsGGebNAtgBkXEeQnL\n/JmkP2tqmzQLIFdhew+nTgEkKaw3Ag3iK+oAUEWyAHIVtveQLAAkKaw3Ag0qbO8hWQBIUlhvBBrE\n2RAAqKJZAEjCYQiQq7C9Z2iysL2mf7Wd7f0Lf/76NAoD0C0pvfExSb8ZEVttHy7pa7Y3RcR4l+YA\nZh3J4qki4p6I2Np//pCknZKOnXRhALplpN7Yv5rwSyRtnkQxwEwp7NRpcrPoH4JcI+kd/YTxhOs2\nbHvi+fPmj9bz5o9prEBgUr7w+dAXvxBtlzEzHDH8j2X7YEn/IGljRFy87L34QJw/ofImY07cN2Qa\nZu2+IYcful8R4ZRlbUd8ZdIVLdneGUqubVJSzoZY0uWSdixvFADKkXIY8lJJ50u6xfbN/Z+9OyI+\nM7mygBlQ2NmQob9uRNwoZnoCxaMJAEhSWJACGlTY3kOyAJCksN4INKiwSVkkCwBJSBZArsL2HpIF\ngCSF9UagQYXtPSQLAEkK641AgzgbAgBVJAsgV2F7D8kCQBKaBYAkhQUpoEGF7T0kCwBJCuuNQIMK\n23sa+XX3+KomVjM1G17edgUZrmu7gNEdtnV/2yWgQYX1RqA5waQsAKgiWQCZ9hW295AsACQprDcC\nzSFZAECNwnoj0JzFuWn+W9v+aWiSBYAkNAsASTgMATLtWzXN3ef7U9xWPZIFgCQkCyDTvrmy5nuT\nLAAkIVkAmfYVdnlvkgWAJCQLINMiyQIAqkgWQKZ9he0+JAsASWgWAJKUlaOABnHqFABqkCyATCQL\nAKhBsgAykSwAoAbNAsi0qLmpPerYXmf7Ntt32H5nzftH2f6M7a22v2H7l8f5fWkWwAyyPSfpEknr\nJD1f0nm2T1y22HpJN0fEiyXNS3qf7eyhB8YsgEwtT/c+TdKdEXGXJNm+WtJrJO1cssz/lfSi/vMj\nJH0nIhZzN0izAGbTcZJ2LXm9W9Lpy5a5TNI/275b0jMk/fw4G6RZAJkmeTbkpoVHtGXhkUGLRMJq\nLpS0NSLmbT9H0ibbJ0XEv+fURLMAOujU+dU6dX71E6//90X3L19kj6Q1S16vUS9dLHWmpP8mSRHx\nTdv/Kul5krbk1MQAJzCbtkg6wfZa24dIeoOk65Ytc5uksyXJ9tHqNYpv5W6QZAFkanNSVkQs2l4v\n6QZJc5Iuj4idtt/Wf/9SSX8k6Qrb29QLBr8TEZWIkopmAcyoiNgoaeOyn1265Pm3Jb2qqe3RLIBM\nXIMTAGqQLIBMXIMTAGqU1RqBBvEVdQCoQbIAMpEsAKAGzQJAkkYOQz4cO4cv1CGffco3e2fDjZef\n03YJozu87QJG5ZGW5jAEAGowwAlkYro3ANQgWQCZmO4NADXKao1AgzgbAgA1SBZAJpIFANQgWQCZ\nmGcBADVoFgCScBgCZGJSFgDUKKs1Ag3i1CkA1CBZAJlIFjVsz9m+2fanJl0QgG5KTRbvkLRD0jMm\nWAswU5iUtYzt4yWdK+mDGvUihQAOGCnJ4s8l/bakIyZcCzBTSptnMfC3tf1KSfdGxM2251da7oEN\nlzzx/ND503To/GmNFQhMzPYFacdC21XMjGGt8UxJr7Z9rqRDJR1h+8qIeNPShY7csH5S9QGT84L5\n3uNxH7+orUpmwsBmEREXSrpQkmy/TNJvLW8UQKk4dTpYTKQKAJ2XPEITEZ+X9PkJ1gLMFJIFANQo\n69wP0CCSBQDUIFkAmZjuDQA1SBZAptKme5MsACQpqzUCDeJsCADUoFkASMJhCJCJwxAAqEGyADIx\nKQsAapAsgExMygKAGmW1RqBBnA0BgBqNJIu7PnxiE6uZmofPm70eedgr9rddwuiuaLuA0Yx6By2S\nBQDUYMwCyESyAIAaNAsASTgMATIx3RsAapAsgExM9waAGmW1RqBBnDoFMBNsr7N9m+07bL9zhWX+\nsv/+NtsvGWd7JAsgU5vJwvacpEsknS1pj6SbbF8XETuXLHOupP8cESfYPl3S+yWdkbtNkgUwm06T\ndGdE3BURj0m6WtJrli3zakkfkaSI2CzpSNtH526QZAFkanmexXGSdi15vVvS6QnLHC9pb84GSRbA\nbIrE5ZZ/mTb1cxUkC6CD7l64Q3cv3DlokT2S1ix5vUa95DBomeP7P8tCswAyTXJS1tHzJ+ro+Sev\nE/P1i25YvsgWSSfYXivpbklvkHTesmWuk7Re0tW2z5D0QERkHYJINAtgJkXEou31km6QNCfp8ojY\naftt/fcvjYhP2z7X9p2SHpZ0wTjbpFkAmdqelBURGyVtXPazS5e9Xt/U9hjgBJCEZAFkajtZTBvJ\nAkASkgWQiWQBADVIFkAmLqsHADVIFkAmLqsHADVoFgCSlJWjgAZx6hQAapAsgEwkCwCoQbIAMjEp\nCwBqkCyATEzKAoAaZbVGoEGcDQGAGjQLAEk4DAEycRgCADVIFkAmJmUBQI1mksUZjaxlav79ac9o\nu4SRHfbog22XMLq/a7uAyWJSFgDUKKs1Ag3ibAgA1CBZAJlIFgBQg2QBZCJZAEANmgWAJByGAJmY\n7g0ANUgWQCamewNAjbJaI9AgTp0CQA2SBZCJZAEANUgWQCbmWSxj+0jb19jeaXuH7Rm7LhaAJqQk\ni7+Q9OmIeL3tVZIOm3BNADpoYLOw/UxJPxERb5akiFiUNIMXgwSax6Ssp/oRSffZvsL2121fZnv1\nNAoD0C3DmsUqSSdL+quIOFnSw5LeNfGqgBmwT3NTe3TBsBy1W9LuiLip//oa1TSLDZc8+Xz+tN4D\n6LqFh6SFh9uuYnYMbBYRcY/tXbafGxG3Szpb0vbly21YP6nygMmZP7z3eNxF9432+a78iz8tKSM0\nb5f0UduHSPqmpAsmWxKALhraLCJim6RTp1ALMFP27S8rWTDdG0CSsk4UAw1aXCRZAEAFyQLItG+x\nrN2HZAEgCc0CQJKychTQoH0McAJAFckCyESyAIAaJAsg0+JjJAsAqCBZAJn27ytr9yFZAAcY28+y\nvcn27bb/0faRKyz3btvbbd9q+29sP23QemkWQK7Fuek9RvMuSZsi4rmSPquaq9vZXivprZJOjogX\nSpqT9AuDVkqzAA48r5b0kf7zj0h6bc0y35X0mKTV/Vt8rJa0Z9BKyzroAprU3XkWR0fE3v7zvZKO\nXr5ARNxv+32S/k3S9yTdEBH/NGilNAtgBtneJOmYmrd+d+mLiAjbUfP550j6DUlr1bsX0N/ZfmNE\nfHSlbTbSLOKlTaxlenz/7N0n6Y62C8gwMNNisK8sSJsXVnw7Is5Z6T3be20f07/g9rMl3Vuz2CmS\nvhQR3+l/5hOSzpQ02WYBFGnRk1v3KWf1Ho/7yz8c5dPXSXqzpPf2//eTNcvcJun3bT9d0qPqXbn/\nq4NWygAncOD575LOsX27pJf3X8v2sbavl564EPeVkrZIuqX/uQ8MWqkjKoczI7Ed+5811iqm7t77\n265gdN9tu4AMs3YYcpakiEiKC7ZD28fbd0byAifXNikkCwBJGLMAci22XcB0kSwAJCFZALlIFgBQ\nRbIAcj3WdgHTRbIAkIRmASAJhyFArn1tFzBdJAsASUgWQC5OnQJAFckCyEWyAIAqkgWQi2QBAFUk\nCyAXyQIAqkgWQC6SBQBU0SwAJOEwBMjFYQgAVJEsgFxcKQsAqkgWQC4ufgMAVSQLIBdnQwCgimQB\n5CJZAEAVzQJAEg5DgFwchgBAFckCyEWyAIAqkgWQi2QBAFUkCyAXyQIAqkgWQK7CLn7TSLM46P5o\nYjXTc0rbBWR4bdsFZLin7QJGdInbrqDTSBZALi5+AwBVNAsASTgMAXJx6hQAqkgWQC6SBQBUkSyA\nXCQLAKgiWQC5CpvuTbIAkIRkAeRiujcAVNEsACThMATIxalTAKgamixsv1vS+ZL2S7pV0gUR8R+T\nLgzoPJLFk2yvlfRWSSdHxAslzUn6hcmXBaBrhiWL76o39WS17X2SVkvaM/GqgFnApKwnRcT9kt4n\n6d8k3S3pgYj4p2kUBqBbhh2GPEfSb0haK+lYSYfbfuMU6gK6b98UHx0w7DDkFElfiojvSJLtT0g6\nU9JHn7rYhiXP5/sPoON2L0h7FtquYmYMaxa3Sfp920+X9KiksyV9tbrYhqbrAibv+Pne43E3XTTa\n5zkb8qSI2CbpSklbJN3S//EHJl0UgHy2f872dtv7bJ88YLkjbV9je6ftHbbPGLTeofMsIuJPJP1J\nRs3Aga27yeJWSa+TdOmQ5f5C0qcj4vW2V0k6bNDCTPcGDjARcZsk2SvfYc32MyX9RES8uf+ZRUkP\nDlov072BMv2IpPtsX2H767Yvs7160AdIFkCuSU7KundBum9hxbdtb5J0TM1bF0bEpxK2sErSyZLW\nR8RNti+W9C5J7xn0AQBd80Pzvcfjdj71TE1EnDPmFnZL2h0RN/VfX6Nes1gRzQLI1ZHJUkPUDlxE\nxD22d9l+bkTcrt60iO2DVsSYBXCAsf0627sknSHpetsb+z8/1vb1SxZ9u6SP2t4m6UWS/mjQekkW\nQK6OnjqNiGslXVvz87sl/cyS19sknZq6XpIFgCQkCyBXR5PFpJAsACQhWQC5uPgNAFTRLAAk4TAE\nyDUbk7IaQ7IAkIRkAeTi1CkAVJEsgFyFJYuGmsXmZlYzLVtmsEduqblOcuf9UNsFoEEzuNcAHcGk\nLACoIlkAuZhnAQBVJAsgV2FnQ0gWAJLQLAAk4TAEyMVhCABUkSyAXEzKAoAqkgWQi0lZAFBFsgBy\ncTYEAKpIFkAukgUAVJEsgFzMswCAKpoFgCQchgC5mJQFAFUkCyAXp04BoIpkAeQiWQBAFckCyMWk\nLACoIlkAuZhnAQBVNAsASTgMAXJF2wVMF8kCQBKaBYAkNAsASWgWAJLQLAAkoVkASEKzAJCEeRZA\ntrK+SdbxZPG1tgsY0Za2C8hwe9sFZPhG2wUUqePN4uttFzCiWWtu0mw2i+1tF9C3OMVH+zreLAB0\nBc0CQBJHjPdtGNuFfZ0GB7KIcMpyvf/uH5x0OUs8M7m2SRn7bEjbvwCA6eDUKZCtGwOP08KYBYAk\nJAsgG5OyAKCCZAFkI1kAQAXJAsjWzbMhtv9U0islfV/SNyVdEBG1k0Jsz6n3pabdEfGqQeslWQAH\nnn+U9IKIOEm9L/+8e8Cy75C0QwnXKqdZAAeYiNgUEfv7LzdLOr5uOdvHSzpX0gclDZ1cyWEIkG0m\nBjjfIuljK7z355J+W9IRKSuiWQAzyPYmScfUvHVhRHyqv8zvSvp+RPxNzedfKeneiLjZ9nzKNmkW\nQLZJDnB+tf+oFxHnDPq07V9W7xDjp1ZY5ExJr7Z9rqRDJR1h+8qIeNOK6xz3W6dAiXrfOt0xxS0+\nf5RvxK6T9D5JL4uIbycs/zJJvzXsbAjJAsjW2TGL/ynpEEmbbEvSlyPi12wfK+myiPiZms8MTQ0k\nCyBDL1lsm+IWT2r9chAkCyBbNydlTQrzLAAkIVkA2To7ZjERJAsASUgWQDbGLACggmYBIAmHIUA2\nBjgBoIJkAWRjgBMAKkgWQDbGLACggmQBZGPMAgAqSBZANsYsAKCCZgEgCYchQDYOQwCggmQBZOPU\nKQBUkCyAbIxZAEAFyQLIxpgFAFSQLIBsjFkAQAXJAsjGmAUAVNAsACThMATIxgAnAFSQLIBsDHAC\nQAXJAsjGmAUAVJAsgGyMWQBAhSOi7RqAmWN76jtORHja21yKZgEgCYchAJLQLAAkoVkASEKzAJCE\nZgEgyf8HmzwGuPTq2GoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa0741e8910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = brain.layers[0].Ws[0].eval()\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.matshow(x)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 11.01352692,  11.28201485,  12.03692055,  12.26954937], dtype=float32)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brain.input_layer.b.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2.0"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.collect_reward(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.39407934,  2.791605  ,  0.92194436,  1.73143704, -1.        ])"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.concatenate([observation, np.array([-1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_prev_frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performances[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "performances_1 = performances[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89357223079637949"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.average(performances[-1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
